import re
from collections import namedtuple

# 字符串令牌解析，有一个字符串想从左至右将其解析为一个令牌流
def demo_1():
    text = 'foo = 23 + 42 * 10'
    NAME = r'(?P<NAME>[a-zA-Z_][a-zA-Z_0-9]*)'
    NUM = r'(?P<NUM>\d+)'
    PLUS = r'(?P<PLUS>\+)'
    TIMES = r'(?P<TIMES>\*)'
    EQ = r'(?P<EQ>=)'
    WS = r'(?P<WS>\s+)'
    master_pat = re.compile('|'.join([NAME, NUM, PLUS, TIMES, EQ, WS]))
    for tok in generate_tokens(master_pat, text):
        print(tok)
    tokens = (tok for tok in generate_tokens(master_pat, text) if tok.type != 'WS')
    for tok in tokens:
        print("use filter: {}".format(tok))

def generate_tokens(pat, text):
    # scanner方法会创建一个scanner对象，在这个对象上不断的调用match方法会一步步的扫描目标文本，每步一个匹配
    # 必须确认使用正则指定了所有输入中可能出现的文本序列，如果有任何不可匹配的文本出现，扫描就会直接停止
    Token = namedtuple('Token', ['type', 'value'])
    scanner = pat.scanner(text)
    for m in iter(scanner.match, None):
        yield Token(m.lastgroup, m.group())

if __name__ == '__main__':
    demo_1()